package generator.driver;

import generator.filedeal.ReadFileDeal;
import generator.groupclass.*;
import generator.statistics.GetClasses;
import generator.statistics.GetTwoSets;
import generator.statistics.NumericalSet;
import generator.statistics.SetCurrentDistributions;
import generator.structure.Pair;
import generator.thread.*;

import java.io.*;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.CountDownLatch;

import javax.naming.directory.DirContext;
 
public class GeneratorStatistics {
	
	/**
	 * numericalSet class that should be initialized here to give the number of partitions
	 */
	public static NumericalSet numericalSet;
	
	/**
	 * have not been used.
	 *  there are some files in one folder, so use this to specify.
	 */
	int whichFile;

	
	/**
	 * specify some special columns.
	 * these should be extracted from scheme file.
	 */
	int primaryKey = 0;
//	int[] timeCols = {1, 2, 3, 4, 5, 6, 8};
//	int[] numericalS = {7,9};
//	int[] hasCompute = {8};
	int[] timeCols = {};
	int[] numericalS = {7,19,20};
	int[] hasCompute = {};
	
	/**
	 * record something
	 */
	int days;
	int dayRows;
	String dataDir_;
//	long Rows;
	
	/**
	 * mark used to split every word
	 * filefoler that generated data live in
	 * buffer
	 */
	String jointMark = ",";
	public String[][] ResultBuffer = new String[3000][20];

	/**
	 * after the first scan, two sets are extracted
	 */
	Set<Integer> candidate = new HashSet<Integer>();
	Set<Integer> Uncandidate = new HashSet<Integer>();
	
	/**
	 * pairs in candicate setv
	 */
	Set<Pair<Integer, Integer>> combineSet = new HashSet<Pair<Integer, Integer>>();

	/**
	 * accumulation information
	 */
	Map<Integer, ArrayList<Pair<String, Integer>>> uncandiAccum = new HashMap<Integer, ArrayList<Pair<String, Integer>>>();

	Map<Pair<Integer, Integer>, ArrayList<Pair<Pair<String, String>, Integer>>> mutualInfoAccum = new HashMap<Pair<Integer, Integer>, ArrayList<Pair<Pair<String, String>, Integer>>>();

	Map<ArrayList<Integer>, ArrayList<Pair<ArrayList<String>, Integer>>> CircleInfoAccum = new HashMap<ArrayList<Integer>, ArrayList<Pair<ArrayList<String>, Integer>>>();

	Map<ArrayList<Integer>, Pair<ArrayList<Pair<Pair<String, String>, Integer>>, ArrayList<HashMap<String, ArrayList<Pair<Pair<String, String>, Integer>>>>>> OverlapInfoAccum = new HashMap<ArrayList<Integer>, Pair<ArrayList<Pair<Pair<String, String>, Integer>>, ArrayList<HashMap<String, ArrayList<Pair<Pair<String, String>, Integer>>>>>>();

	/**
	 * in order to keep the order as in the primary table
	 */
	HashMap<Integer, String> record = new HashMap<Integer, String>();

	public static Mutual mutual;
	public static Uncandidate ucandi;
	public static Overlap overlap;
	public static Circle circle;
	Runtime rt = Runtime.getRuntime();
	
	public GeneratorStatistics(int generateDays, int dayRows, String FileSampleString, 
						String FileInput_, int numericalPartition, int topicNumbers, 
						int lpDimension, String theta, String phi, String key, 
						String fast, String lp, String lpTemp, String dir,
						int filter, int upperLimit, double threshold, int numInGroup, double size) {
			
		System.out.println(FileSampleString);
		this.days = generateDays;
		this.dataDir_ = FileInput_;
		
		this.numericalSet = new NumericalSet(numericalPartition);
				
		File fileSample = new File(FileSampleString);
		
		this.dayRows = dayRows;
		
		File files = new File(dataDir_); 
		File[] file = files.listFiles();
		
		long tableRows;
		int tableCol;
		
		for(File fileName : file) {
			
			int currentDays = fileName.listFiles().length;
			
			//initial the groups classes
			mutual = new Mutual(currentDays, generateDays, dayRows, theta, phi, lp, lpTemp, key, topicNumbers, lpDimension);
			circle = new Circle(currentDays, generateDays, dayRows, theta, phi, lp, lpTemp, key, topicNumbers, lpDimension);
			overlap = new Overlap(currentDays, generateDays, dayRows, theta, phi, lp, lpTemp, key, topicNumbers, lpDimension);
			ucandi = new Uncandidate();
			
			//give fileReader arguments
			ReadFileDeal fileReader = new ReadFileDeal(fileName, fileSample, jointMark, primaryKey, timeCols, numericalS, hasCompute);
			
			System.out.println("#########################  " + fileName + "  #####################################");
			
			GetTwoSets twoSets = new GetTwoSets(fileReader, numericalS, numericalPartition, filter, upperLimit, threshold, numInGroup);

			SimpleDateFormat df0 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
			System.out.println("initial all classes: " + df0.format(new Date()));
			
			long start = System.currentTimeMillis();
			twoSets.genTwoSets();
	
//			twoSets.roughClassify2();
			this.Uncandidate = twoSets.getSingleColumnSet();
//			this.Uncandidate = new HashSet<Integer>();
			
			this.combineSet = twoSets.getCombineDataSet();
			
			tableRows = twoSets.getRows();
			tableCol = twoSets.getCols();
			long end = System.currentTimeMillis();
			System.out.println("getTwoSets: Used Memory = " + (rt.totalMemory() - rt.freeMemory()) + " has used time " + (end - start));
			
			System.out.println("combineSet: " + combineSet);
			System.out.println("uncandidate: " + Uncandidate);
			System.out.println("sample file Rows: " + tableRows + " number of columns: " + tableCol);
			
			
			
			//twoSets.getRetent();
			
			//twoSets = null;
			System.gc();
			
			System.out.println("After getTwoSets: Used Memory = " + (rt.totalMemory() - rt.freeMemory()));
			
			System.out.println("Have found uncandidate and candidate: " + df0.format(new Date()));
			
			System.out.println("dataGenerator");
			
			fileReader.setNumericalClass(numericalSet);
			fileReader.readData("c");
			
			GetClasses assignGroup = new GetClasses();  //modify the three class' elements
			
			if(combineSet.size() != 0) {
				assignGroup.classify(combineSet);
			}
			
			/*ArrayList<Integer> arrayList = new ArrayList<Integer>();
			ArrayList<ArrayList<Integer>> overlap = new ArrayList<ArrayList<Integer>>();
			arrayList = new ArrayList<Integer>();
			arrayList.add(9);
			arrayList.add(11);
			arrayList.add(18);
			overlap.add(arrayList);
			GeneratorStatistics.overlap.setGroup(overlap);*/
			
			//circle
//			ArrayList<ArrayList<Integer>> circle = new ArrayList<ArrayList<Integer>>();
			
//			ArrayList<Integer> arrayList = new ArrayList<Integer>();
//			arrayList.add(12);
//			arrayList.add(19);
//			arrayList.add(11);
//			arrayList.add(10);
//			circle.add(arrayList);
			
			//mutual
//			ArrayList<ArrayList<Integer>> MUTUAL = new ArrayList<ArrayList<Integer>>();
//			arrayList = new ArrayList<Integer>();
//			arrayList.add(12);
//			arrayList.add(18);
//			MUTUAL.add(arrayList);
			
			//overlap
//			ArrayList<ArrayList<Integer>> overlap = new ArrayList<ArrayList<Integer>>();
			
//			arrayList = new ArrayList<Integer>();
//			arrayList.add(9);
//			arrayList.add(11);
//			arrayList.add(18);
//			overlap.add(arrayList);
			
//			GeneratorStatistics.mutual.setGroup(MUTUAL);
//			GeneratorStatistics.circle.setGroup(circle);
//			GeneratorStatistics.overlap.setGroup(overlap);
			
			
			start = System.currentTimeMillis();
			SetCurrentDistributions scd = new SetCurrentDistributions(fileReader, size);
			scd.setMutualDistributions();
			scd.setCircleDistributions();
			scd.setOverlapDistributions();
			//System.out.println("scd.getUncandiAccum:");
			scd.getUncandiAccum(Uncandidate);
			
			end = System.currentTimeMillis();
//			mutual.listLength = 0;
//			overlap.listLength = 0;
//			circle.listLength = 0;
			System.out.println("set the distribution has used " + (end - start));
			System.out.println("uncandiAccum_accumulation: Used Memory = " + (rt.totalMemory() - rt.freeMemory()));
			
			System.gc();
			System.out.println("After class accumulation: Used Memory = " + (rt.totalMemory() - rt.freeMemory()));
			System.exit(1);
			
			int columnNumbers = fileReader.getCols();
			
			String structureFile = dir + "/structure.txt";
			writeStructure(structureFile, columnNumbers, currentDays, topicNumbers, lpDimension);
			
			seriablizeNumerical(dir);
			seriablizeUncandidate(dir);
			seriablizeOverlapLatter(dir);
			
			start = System.currentTimeMillis();
			
			if(fast.equals("fast")) {
				
//				mutual.LDAfastCompute();
//				circle.LDAfastCompute();
//				overlap.LDAfastCompute();

				parallelLDAcompute(1);
			}
			else {
				
//				mutual.LDAcompute();
//				circle.LDAcompute();
//				overlap.LDAcompute();

				parallelLDAcompute(0);
			}
			
			end = System.currentTimeMillis();
			System.out.println("all LDA has used time " + (end - start));
			System.out.println("After LDA and dependency: Used Memory = " + (rt.totalMemory() - rt.freeMemory()));
			
		}

	}
	
	private void seriablizeOverlapLatter(String dir) {
		String overlapLatterString = dir + "/overlapLatterAccum.txt";
		
		Map<ArrayList<Integer>, ArrayList<HashMap<String, ArrayList<Pair<Pair<String, String>, Integer>>>>> overlapLatter = 
				overlap.getLatterAccumulation();
		try {
			ObjectOutputStream os = new ObjectOutputStream(new FileOutputStream(overlapLatterString));
			os.writeObject(overlapLatter);
			os.close();
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}
	}

	private void seriablizeNumerical(String dir) {
		String numericalOut = dir + "/numericalClass.txt";
		
		try {
			ObjectOutputStream os = new ObjectOutputStream(new FileOutputStream(numericalOut));
			os.writeObject(numericalSet);
			os.close();
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}
	}
	
	private void seriablizeUncandidate(String dir) {
		String uncandiOut = dir + "/uncandidateClass.txt";
		
		try {
			ObjectOutputStream os = new ObjectOutputStream(new FileOutputStream(uncandiOut));
			os.writeObject(ucandi);
			os.close();
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}
	}

	private void writeStructure(String structureFile, int columnNumbers, int fileNum, int topicNumbers, int lpDimension) {

		try {
			PrintWriter pw = new PrintWriter(new FileWriter(structureFile));
			
			//write the structure
//			int length = ucandi.getListLength();
//			int[] uncandiKey = ucandi.getKeys();
//			pw.println("uncandidate:" + length);
//			for(int i = 0; i < length; i++) {
//				pw.println(uncandiKey[i]);
//			}
//			
//			pw.println();
			
			pw.println(columnNumbers + " " + fileNum + " " + topicNumbers + " " + lpDimension);
			pw.println();
			
			int length = mutual.getListLength();
			ArrayList<ArrayList<Integer>> groups = mutual.getGroups();
			pw.println("mutual:" + length);
			for(int i = 0; i < length; i++) {
				ArrayList<Integer> oneGroup = groups.get(i);
				String temp = "";
				
				for(int j = 0; j < oneGroup.size(); j++) {
					temp += oneGroup.get(j) + " ";
				}
				
				pw.println(temp.substring(0,temp.length()-1));
			}
			
			pw.println();
			
			length = circle.getListLength();
			groups = circle.getGroups();
			pw.println("circle:" + length);
			for(int i = 0; i < length; i++) {
				ArrayList<Integer> oneGroup = groups.get(i);
				String temp = "";
				
				for(int j = 0; j < oneGroup.size(); j++) {
					temp += oneGroup.get(j) + " ";
				}
				
				pw.println(temp.substring(0,temp.length()-1));
			}
			
			pw.println();
			
			length = overlap.getListLength();
			groups = overlap.getGroups();
			pw.println("overlap:" + length);
			for(int i = 0; i < length; i++) {
				ArrayList<Integer> oneGroup = groups.get(i);
				String temp = "";
				
				for(int j = 0; j < oneGroup.size(); j++) {
					temp += oneGroup.get(j) + " ";
				}
				
				pw.println(temp.substring(0,temp.length()-1));
			}
			pw.close();
			
		} catch (IOException e) {
			e.printStackTrace();
		}
	}

	public void parallelLDAcompute(int isFast) {
		
		int groupNumbers = mutual.getListLength() + circle.getListLength()+ overlap.getListLength();
		CountDownLatch latch = new CountDownLatch(groupNumbers);
		
		int ldaFlag = 1;
		
		for(int i = 0; i < mutual.getListLength(); i++) {
			
			MutualThread mutualThread = new MutualThread(latch, isFast, ldaFlag, i);
			new Thread(mutualThread).start();
		}
		
		for(int i = 0; i < circle.getListLength(); i++) {
			
			CircleThread circleThread = new CircleThread(latch, isFast, ldaFlag, i);
			new Thread(circleThread).start();
		}
		
		for(int i = 0; i < overlap.getListLength(); i++) {
			
			OverlapThread overlapThread = new OverlapThread(latch, isFast, ldaFlag, i);
			new Thread(overlapThread).start();
		}
		
		try {
			latch.await();
		} catch (InterruptedException e) {
			e.printStackTrace();
		}
		
	}
	
	public static void main(String[] args){
		
		System.out.println("input arguments file, folder");
		
		SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
		System.out.println("Start: " + df.format(new Date()));
		
		String inputArguments = args[0];
		String dir = args[1];
		
		System.out.println("the position of inputArguments: " + inputArguments);
		
		int days = 0;
		int dayRows = 0;
		String sampleFile = "";
		String inputFolder = "";
		//String inputFolder = args[2];
		int numericalPartition = 0;
		int topicNumbers = 0;
		//int topicNumbers = Integer.parseInt(args[2]);
		int lpDimension = 0;
		String fast = "";
		int filter = 18;         //prune the columns that has few distinct numbers
		int upperLimit = 13000;  //the max distinct in each group
		double threshold = 0.08; //p_value
		//double threshold = Double.parseDouble(args[2]);
		int numInGroup = 1000000;
		double datasize = 1;  //sample of the real data
		
		try {
			BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(inputArguments)));
			days = Integer.parseInt((br.readLine().split("="))[1]);
			dayRows = Integer.parseInt((br.readLine().split("="))[1]);
			sampleFile = (br.readLine().split("="))[1];
			inputFolder = (br.readLine().split("="))[1];
			numericalPartition = Integer.parseInt((br.readLine().split("="))[1]);
			topicNumbers = Integer.parseInt((br.readLine().split("="))[1]);
			lpDimension = Integer.parseInt((br.readLine().split("="))[1]);
			fast = (br.readLine().split("="))[1];
			filter = Integer.parseInt((br.readLine().split("="))[1]);
			//upperLimit = Integer.parseInt((br.readLine().split("="))[1]);
			threshold = Double.parseDouble((br.readLine().split("="))[1]);
			numInGroup = Integer.parseInt((br.readLine().split("="))[1]);
			datasize = Double.parseDouble((br.readLine().split("="))[1]);
			
			br.close();
		} catch (IOException e1) {
			e1.printStackTrace();
		}
		
		//String dir = System.getProperty("user.dir");
		//File dirFile = new File(new File(dir).getParent() + (new File(dir)).separator + "position.txt");
		File dirFile = new File(dir + "/position.txt");
		
		String[] temp = dir.split("\\\\");
		
		if(temp.length >= 2) {
			String tempString = "";
			for(int i = 0; i < temp.length; i++) {
				tempString += temp[i] + "/";
			} // D:/WORKSPACE/position.txt/structure.txt
			dir = tempString.substring(0, tempString.length() - 1);
		}
		
		
		String theta = dir + "/theta";
		String phi = dir + "/phi";
		String lp = dir + "/lp";
		String key = dir + "/key";
		
		String structureFile = dir + "/" + "structure.txt";
		String uncandiFile = dir + "/UncandidateClass.txt";
		String numericalFile = dir + "/" + "numericalClass.txt";
		String overlapLatterAccum = dir + "/" + "overlapLatterAccum.txt";
		
		try {
			PrintWriter pw = new PrintWriter(new FileWriter(dirFile));
			
			pw.println(theta);
			pw.println(phi);
			pw.println(lp);
			pw.println(key);
			pw.println(structureFile);
			pw.println(uncandiFile);
			pw.println(numericalFile);
			pw.println(overlapLatterAccum);
			
			pw.close();
		} catch (IOException e) {
			e.printStackTrace();
		}
		
		String lpTemp_ = dir + "/lptemp";//the interval result in lp
		
		try {
			PrintWriter pw = new PrintWriter(new FileWriter(dirFile, true));
			pw.println(overlapLatterAccum);
			pw.close();
		} catch (IOException e) {
			e.printStackTrace();
		}
		
		long start = System.currentTimeMillis();
		
		GeneratorStatistics generatorStatistics = new GeneratorStatistics(days, 
																		dayRows, 
																		sampleFile, 
																		inputFolder, 
																		numericalPartition, 
																		topicNumbers,
																		lpDimension,
																		theta, phi, key, fast, 
																		lp, lpTemp_,
																		dir, filter, upperLimit, 
																		threshold, numInGroup, datasize);
		
		System.out.println("Have done all!!: " + df.format(new Date()));
		long end = System.currentTimeMillis();
		System.out.println("elapse time: " + (end - start));
	}

}
